<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="HTTP RESTful Interface to the Cask Data Application Platform" name="description" />
<meta content="Copyright © 2015-2017 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Metadata HTTP RESTful API</title>

    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <script type="text/javascript" src="../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../index.html" />
    <link rel="up" title="CDAP HTTP RESTful API v3" href="index.html" />
    <link rel="next" title="Metrics HTTP RESTful API" href="metrics.html" />
    <link rel="prev" title="Logging HTTP RESTful API" href="logging.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: reference-manual -->
<!-- theme_manual_highlight: reference -->
<!-- sidebar_title_link: Reference -->

  <div role="note" aria-label="manuals links"><h3>Reference</h3>
    <ul class="reference-page-menu">
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../index.html"> Introduction</a></li>
<li class="toctree-l1"><a class="reference internal" href="../characters.html"> Supported Characters</a></li>
<li class="toctree-l1"><a class="reference internal" href="../cli-api.html"> Command Line Interface API</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html"> HTTP RESTful API</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="artifact.html">Artifact</a></li>
<li class="toctree-l2"><a class="reference internal" href="configuration.html">Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="dataset.html">Dataset</a></li>
<li class="toctree-l2"><a class="reference internal" href="lifecycle.html">Lifecycle</a></li>
<li class="toctree-l2"><a class="reference internal" href="logging.html">Logging</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#">Metadata</a></li>
<li class="toctree-l2"><a class="reference internal" href="metrics.html">Metrics</a></li>
<li class="toctree-l2"><a class="reference internal" href="monitor.html">Monitor</a></li>
<li class="toctree-l2"><a class="reference internal" href="namespace.html">Namespace</a></li>
<li class="toctree-l2"><a class="reference internal" href="dashboard.html">Dashboard</a></li>
<li class="toctree-l2"><a class="reference internal" href="preferences.html">Preferences</a></li>
<li class="toctree-l2"><a class="reference internal" href="profile.html">Profile</a></li>
<li class="toctree-l2"><a class="reference internal" href="query.html">Query</a></li>
<li class="toctree-l2"><a class="reference internal" href="reports.html">Reports</a></li>
<li class="toctree-l2"><a class="reference internal" href="security.html">Security</a></li>
<li class="toctree-l2"><a class="reference internal" href="service.html">Service</a></li>
<li class="toctree-l2"><a class="reference internal" href="transactions.html">Transactions</a></li>
<li class="toctree-l2"><a class="reference internal" href="workflow.html">Workflow</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../java-client-api.html"> Java Client API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../javadocs/index.html"> Javadocs</a></li>
<li class="toctree-l1"><a class="reference internal" href="../release-notes.html"> 发行说明</a></li>
<li class="toctree-l1"><a class="reference internal" href="../licenses/index.html"> Trademarks, Licenses, and Dependencies</a></li>
<li class="toctree-l1"><a class="reference internal" href="../glossary.html"> 术语表</a></li>
<li class="toctree-l1"><a class="reference internal" href="../genindex.html"> Index</a></li>
</ul>
</nav>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="metadata-http-restful-api">
<span id="http-restful-api-v3-metadata"></span><span id="http-restful-api-metadata"></span><h1>Metadata HTTP RESTful API<a class="headerlink" href="#metadata-http-restful-api" title="Permalink to this headline">🔗</a></h1>
<p>Use the CDAP Metadata HTTP RESTful API to set, retrieve, and delete the metadata annotations
of applications, datasets, streams, and other entities in CDAP.</p>
<p><strong>Note</strong>: Metadata for versioned entities is not versioned, including entities such as applications,
programs, schedules, and program runs. Additions to metadata in one version are reflected in all versions.</p>
<p>Metadata consists of <strong>properties</strong> (a list of key-value pairs) or <strong>tags</strong> (a list of keys).
Metadata and their use are described in the <a class="reference external" href="../../../developer-manual/metadata/discovery-lineage.html#metadata-lineage" title="(in Cask Data Application Platform v6.1.1)"><span class="xref std std-ref">开发手册: Metadata and Lineage</span></a>.</p>
<p>The HTTP RESTful API is divided into these sections:</p>
<ul class="simple">
<li><a class="reference internal" href="#http-restful-api-metadata-properties"><span class="std std-ref">metadata properties</span></a></li>
<li><a class="reference internal" href="#http-restful-api-metadata-tags"><span class="std std-ref">metadata tags</span></a></li>
<li><a class="reference internal" href="#http-restful-api-metadata-searching"><span class="std std-ref">searching metadata</span></a></li>
<li><a class="reference internal" href="#http-restful-api-metadata-lineage"><span class="std std-ref">viewing lineage</span></a></li>
<li><a class="reference internal" href="#http-restful-api-metadata-fieldlevellineage"><span class="std std-ref">field level lineage</span></a></li>
<li><a class="reference internal" href="#http-restful-api-metadata-run"><span class="std std-ref">metadata for a run of a program</span></a></li>
</ul>
<p>Metadata keys, values, and tags must conform to the CDAP <a class="reference internal" href="../characters.html#supported-characters"><span class="std std-ref">alphanumeric extra extended
character set</span></a>, and are limited to 50 characters in length. The entire
metadata object associated with a single entity is limited to 10K bytes in size.</p>
<p>There is one reserved word for property keys and values: <em>tags</em>, either as <code class="docutils literal notranslate"><span class="pre">tags</span></code> or
<code class="docutils literal notranslate"><span class="pre">TAGS</span></code>. Tags themselves have no reserved words.</p>
<p>The online metadata is migrated when CDAP is upgraded from 4.3.x to 5.0.0. During metadata migration,
writes to non-migrated entities fail and reads on metadata for non-migrated entities do not have all the metadata.
Metadata migration takes approximately 2-3 minutes. You can check the metadata migration status by using the
:ref:upgrade status &lt;http-restful-api-monitor-status-system-upgrade&gt; REST endpoint.</p>
<p>All methods or endpoints described in this API have a base URL (typically
<code class="docutils literal notranslate"><span class="pre">http://&lt;host&gt;:11015</span></code> or <code class="docutils literal notranslate"><span class="pre">https://&lt;host&gt;:10443</span></code>) that precedes the resource
identifier, as described in the <a class="reference internal" href="introduction.html#http-restful-api-conventions-base-url"><span class="std std-ref">RESTful API Conventions</span></a>.
These methods return a status code, as listed in the <a class="reference internal" href="introduction.html#http-restful-api-status-codes"><span class="std std-ref">RESTful API  Status Codes</span></a>.</p>
<div class="section" id="metadata-properties">
<span id="http-restful-api-metadata-properties"></span><h2>Metadata Properties<a class="headerlink" href="#metadata-properties" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="annotating-properties">
<h3>Annotating Properties<a class="headerlink" href="#annotating-properties" title="Permalink to this headline">🔗</a></h3>
<p>To annotate user metadata properties for an application, dataset, or other entities including custom entities,
submit an HTTP POST request:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">POST /v3/namespaces/&lt;namespace-id&gt;/&lt;entity-details&gt;/metadata/properties</span>
</pre></div>
</div>
<p>or, for a particular program of a specific application:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">POST /v3/namespaces/&lt;namespace-id&gt;/apps/&lt;app-id&gt;/&lt;program-type&gt;/&lt;program-id&gt;/metadata/properties</span>
</pre></div>
</div>
<p>or, for a particular version of an artifact:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">POST /v3/namespaces/&lt;namespace-id&gt;/artifacts/&lt;artifact-id&gt;/versions/&lt;artifact-version&gt;/metadata/properties</span>
</pre></div>
</div>
<p>or, for a particular view of a stream:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">POST /v3/namespaces/&lt;namespace-id&gt;/streams/&lt;stream-id&gt;/views/&lt;view-id&gt;/metadata/properties</span>
</pre></div>
</div>
<p>or, for a custom entity like field of a dataset:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">POST /v3/namespaces/&lt;namespace-id&gt;/datasets/&lt;dataset-id&gt;/field/&lt;field-name&gt;/metadata/properties</span>
</pre></div>
</div>
<p>with the metadata properties as a JSON string map of string-string pairs, passed in the
request body:</p>
<div class="highlight-json-ellipsis notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
  <span class="nt">&quot;key1&quot;</span> <span class="p">:</span> <span class="s2">&quot;value1&quot;</span><span class="p">,</span>
  <span class="nt">&quot;key2&quot;</span> <span class="p">:</span> <span class="s2">&quot;value2&quot;</span><span class="p">,</span>
  <span class="c">...</span>
<span class="p">}</span>
</pre></div>
</div>
<p>New property keys will be added and existing keys will be updated.
Existing keys not in the properties map will not be deleted.</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">namespace-id</span></code></td>
<td>Namespace ID</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">entity-details</span></code></td>
<td><a class="reference internal" href="#metadata-custom-entities"><span class="std std-ref">Hierarchical key-value representation of the entity</span></a></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">app-id</span></code></td>
<td>Name of the application</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">program-type</span></code></td>
<td>One of <code class="docutils literal notranslate"><span class="pre">flows</span></code>, <code class="docutils literal notranslate"><span class="pre">mapreduce</span></code>, <code class="docutils literal notranslate"><span class="pre">spark</span></code>, <code class="docutils literal notranslate"><span class="pre">workflows</span></code>, <code class="docutils literal notranslate"><span class="pre">services</span></code>, or <code class="docutils literal notranslate"><span class="pre">workers</span></code></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">program-id</span></code></td>
<td>Name of the program</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">artifact-id</span></code></td>
<td>Name of the artifact</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">artifact-version</span></code></td>
<td>Version of the artifact</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">stream-id</span></code></td>
<td>Name of the stream</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">view-id</span></code></td>
<td>Name of the stream view</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">dataset-id</span></code></td>
<td>Name of the dataset</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">field-name</span></code></td>
<td>Name of the field</td>
</tr>
</tbody>
</table>
<p class="rubric">HTTP Responses</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Status Codes</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">200</span> <span class="pre">OK</span></code></td>
<td>The properties were set</td>
</tr>
</tbody>
</table>
<p><strong>Note</strong>: When using this API, properties can be added to the metadata of the specified entity
only in the <em>user</em> scope.</p>
</div>
<div class="section" id="retrieving-properties">
<h3>Retrieving Properties<a class="headerlink" href="#retrieving-properties" title="Permalink to this headline">🔗</a></h3>
<p>To retrieve user metadata properties for an application, dataset, or other entities including custom entities,
submit an HTTP GET request:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/&lt;entity-details&gt;/metadata/properties[?scope=&lt;scope&gt;]</span>
</pre></div>
</div>
<p>or, for a particular program of a specific application:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/apps/&lt;app-id&gt;/&lt;program-type&gt;/&lt;program-id&gt;/metadata/properties[?scope=&lt;scope&gt;]</span>
</pre></div>
</div>
<p>or, for a particular version of an artifact:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/artifacts/&lt;artifact-id&gt;/versions/&lt;artifact-version&gt;/metadata/properties[?scope=&lt;scope&gt;]</span>
</pre></div>
</div>
<p>or, for a particular view of a stream:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/streams/&lt;stream-id&gt;/views/&lt;view-id&gt;/metadata/properties[?scope=&lt;scope&gt;]</span>
</pre></div>
</div>
<p>or, for a custom entity like field of a dataset:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/datasets/&lt;dataset-id&gt;/field/&lt;field-name&gt;/metadata/properties[?scope=&lt;scope&gt;]</span>
</pre></div>
</div>
<p>with the metadata properties returned as a JSON string map of string-string pairs, passed
in the response body (pretty-printed):</p>
<div class="highlight-json-ellipsis notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
  <span class="nt">&quot;key1&quot;</span> <span class="p">:</span> <span class="s2">&quot;value1&quot;</span><span class="p">,</span>
  <span class="nt">&quot;key2&quot;</span> <span class="p">:</span> <span class="s2">&quot;value2&quot;</span><span class="p">,</span>
  <span class="c">...</span>
<span class="p">}</span>
</pre></div>
</div>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">namespace-id</span></code></td>
<td>Namespace ID</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">entity-details</span></code></td>
<td><a class="reference internal" href="#metadata-custom-entities"><span class="std std-ref">Hierarchical key-value representation of the entity</span></a></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">app-id</span></code></td>
<td>Name of the application</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">program-type</span></code></td>
<td>One of <code class="docutils literal notranslate"><span class="pre">flows</span></code>, <code class="docutils literal notranslate"><span class="pre">mapreduce</span></code>, <code class="docutils literal notranslate"><span class="pre">spark</span></code>, <code class="docutils literal notranslate"><span class="pre">workflows</span></code>, <code class="docutils literal notranslate"><span class="pre">services</span></code>, or <code class="docutils literal notranslate"><span class="pre">workers</span></code></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">program-id</span></code></td>
<td>Name of the program</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">artifact-id</span></code></td>
<td>Name of the artifact</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">artifact-version</span></code></td>
<td>Version of the artifact</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">stream-id</span></code></td>
<td>Name of the stream</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">view-id</span></code></td>
<td>Name of the stream view</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">dataset-id</span></code></td>
<td>Name of the dataset</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">field-name</span></code></td>
<td>Name of the field</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">scope</span></code></td>
<td>Optional scope filter. If not specified, properties in the <code class="docutils literal notranslate"><span class="pre">user</span></code> and
<code class="docutils literal notranslate"><span class="pre">system</span></code> scopes are returned. Otherwise, only properties in the specified scope are returned.</td>
</tr>
</tbody>
</table>
<p class="rubric">HTTP Responses</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Status Codes</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">200</span> <span class="pre">OK</span></code></td>
<td>The properties requested were returned as a JSON string in the body of the response which can be empty if there are no properties associated with the entity, or the entity does not exist</td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="deleting-properties">
<h3>Deleting Properties<a class="headerlink" href="#deleting-properties" title="Permalink to this headline">🔗</a></h3>
<p>To delete <strong>all</strong> user metadata properties for an application, dataset, or other entities including custom entities,
submit an HTTP DELETE request:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/&lt;entity-details&gt;/metadata/properties</span>
</pre></div>
</div>
<p>or, for all user metadata properties of a particular program of a specific application:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/apps/&lt;app-id&gt;/&lt;program-type&gt;/&lt;program-id&gt;/metadata/properties</span>
</pre></div>
</div>
<p>or, for a particular version of an artifact:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/artifacts/&lt;artifact-id&gt;/versions/&lt;artifact-version&gt;/metadata/properties</span>
</pre></div>
</div>
<p>or, for a particular view of a stream:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/streams/&lt;stream-id&gt;/views/&lt;view-id&gt;/metadata/properties</span>
</pre></div>
</div>
<p>To delete <strong>a specific property</strong> for an application, dataset, or stream, submit
an HTTP DELETE request with the property key:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/&lt;entity-type&gt;/&lt;entity-id&gt;/metadata/properties/&lt;key&gt;</span>
</pre></div>
</div>
<p>or, for a particular property of a program of a specific application:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/apps/&lt;app-id&gt;/&lt;program-type&gt;/&lt;program-id&gt;/metadata/properties/&lt;key&gt;</span>
</pre></div>
</div>
<p>or, for a particular version of an artifact:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/artifacts/&lt;artifact-id&gt;/versions/&lt;artifact-version&gt;/metadata/properties/&lt;key&gt;</span>
</pre></div>
</div>
<p>or, for a particular view of a stream:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/streams/&lt;stream-id&gt;/views/&lt;view-id&gt;/metadata/properties/&lt;key&gt;</span>
</pre></div>
</div>
<p>or, for a custom entity like field of a dataset:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/datasets/&lt;dataset-id&gt;/field/&lt;field-name&gt;/metadata/properties/&lt;key&gt;</span>
</pre></div>
</div>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">namespace-id</span></code></td>
<td>Namespace ID</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">entity-details</span></code></td>
<td><a class="reference internal" href="#metadata-custom-entities"><span class="std std-ref">Hierarchical key-value representation of the entity</span></a></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">app-id</span></code></td>
<td>Name of the application</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">program-type</span></code></td>
<td>One of <code class="docutils literal notranslate"><span class="pre">flows</span></code>, <code class="docutils literal notranslate"><span class="pre">mapreduce</span></code>, <code class="docutils literal notranslate"><span class="pre">spark</span></code>, <code class="docutils literal notranslate"><span class="pre">workflows</span></code>, <code class="docutils literal notranslate"><span class="pre">services</span></code>, or <code class="docutils literal notranslate"><span class="pre">workers</span></code></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">program-id</span></code></td>
<td>Name of the program</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">artifact-id</span></code></td>
<td>Name of the artifact</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">artifact-version</span></code></td>
<td>Version of the artifact</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">stream-id</span></code></td>
<td>Name of the stream</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">view-id</span></code></td>
<td>Name of the stream view</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">dataset-id</span></code></td>
<td>Name of the dataset</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">field-name</span></code></td>
<td>Name of the field</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">key</span></code></td>
<td>Metadata property key</td>
</tr>
</tbody>
</table>
<p class="rubric">HTTP Responses</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Status Codes</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">200</span> <span class="pre">OK</span></code></td>
<td>The method was successfully called, and the properties were deleted, or in the case of a
specific key, were either deleted or the key was not present, or the entity itself was not present</td>
</tr>
</tbody>
</table>
<p><strong>Note</strong>: When using this API, only properties in the <em>user</em> scope can be deleted.</p>
</div>
</div>
<div class="section" id="metadata-tags">
<span id="http-restful-api-metadata-tags"></span><h2>Metadata Tags<a class="headerlink" href="#metadata-tags" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="adding-tags">
<h3>Adding Tags<a class="headerlink" href="#adding-tags" title="Permalink to this headline">🔗</a></h3>
<p>To add user metadata tags for an application, dataset, or other entities including custom entities,
submit an HTTP POST request:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">POST /v3/namespaces/&lt;namespace-id&gt;/&lt;entity-details&gt;/metadata/tags</span>
</pre></div>
</div>
<p>or, for a particular program of a specific application:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">POST /v3/namespaces/&lt;namespace-id&gt;/apps/&lt;app-id&gt;/&lt;program-type&gt;/&lt;program-id&gt;/metadata/tags</span>
</pre></div>
</div>
<p>or, for a particular version of an artifact:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">POST /v3/namespaces/&lt;namespace-id&gt;/artifacts/&lt;artifact-id&gt;/versions/&lt;artifact-version&gt;/metadata/tags</span>
</pre></div>
</div>
<p>or, for a particular view of a stream:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">POST /v3/namespaces/&lt;namespace-id&gt;/streams/&lt;stream-id&gt;/views/&lt;view-id&gt;/metadata/tags</span>
</pre></div>
</div>
<p>or, for a custom entity like field of a dataset:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">POST /v3/namespaces/&lt;namespace-id&gt;/datasets/&lt;dataset-id&gt;/field/&lt;field-name&gt;/metadata/tags</span>
</pre></div>
</div>
<p>with the metadata tags, as a list of strings, passed in the JSON request body:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">[&quot;tag1&quot;, &quot;tag2&quot;]</span>
</pre></div>
</div>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">namespace-id</span></code></td>
<td>Namespace ID</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">entity-details</span></code></td>
<td><a class="reference internal" href="#metadata-custom-entities"><span class="std std-ref">Hierarchical key-value representation of the entity</span></a></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">app-id</span></code></td>
<td>Name of the application</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">program-type</span></code></td>
<td>One of <code class="docutils literal notranslate"><span class="pre">flows</span></code>, <code class="docutils literal notranslate"><span class="pre">mapreduce</span></code>, <code class="docutils literal notranslate"><span class="pre">spark</span></code>, <code class="docutils literal notranslate"><span class="pre">workflows</span></code>, <code class="docutils literal notranslate"><span class="pre">services</span></code>, or <code class="docutils literal notranslate"><span class="pre">workers</span></code></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">program-id</span></code></td>
<td>Name of the program</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">artifact-id</span></code></td>
<td>Name of the artifact</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">artifact-version</span></code></td>
<td>Version of the artifact</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">stream-id</span></code></td>
<td>Name of the stream</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">view-id</span></code></td>
<td>Name of the stream view</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">dataset-id</span></code></td>
<td>Name of the dataset</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">field-name</span></code></td>
<td>Name of the field</td>
</tr>
</tbody>
</table>
<p class="rubric">HTTP Responses</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Status Codes</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">200</span> <span class="pre">OK</span></code></td>
<td>The tags were set</td>
</tr>
</tbody>
</table>
<p><strong>Note</strong>: When using this API, tags can be added to the metadata of the specified entity only in the user scope.</p>
</div>
<div class="section" id="retrieving-tags">
<h3>Retrieving Tags<a class="headerlink" href="#retrieving-tags" title="Permalink to this headline">🔗</a></h3>
<p>To retrieve user metadata tags for an application, dataset, or other entities including custom entities,
submit an HTTP GET request:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/&lt;entity-details&gt;/metadata/tags[?scope=&lt;scope&gt;]</span>
</pre></div>
</div>
<p>or, for a particular program of a specific application:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/apps/&lt;app-id&gt;/&lt;program-type&gt;/&lt;program-id&gt;/metadata/tags[?scope=&lt;scope&gt;]</span>
</pre></div>
</div>
<p>or, for a particular version of an artifact:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/artifacts/&lt;artifact-id&gt;/versions/&lt;artifact-version&gt;/metadata/tags[?scope=&lt;scope&gt;]</span>
</pre></div>
</div>
<p>or, for a particular view of a stream:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/streams/&lt;stream-id&gt;/views/&lt;view-id&gt;/metadata/tags[?scope=&lt;scope&gt;]</span>
</pre></div>
</div>
<p>or, for a custom entity like field of a dataset:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/dataset/&lt;dataset-id&gt;/field/&lt;field-name&gt;/metadata/tags[?scope=&lt;scope&gt;]</span>
</pre></div>
</div>
<p>with the metadata tags returned as a JSON string in the return body:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">[&quot;tag1&quot;, &quot;tag2&quot;]</span>
</pre></div>
</div>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">namespace-id</span></code></td>
<td>Namespace ID</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">entity-details</span></code></td>
<td><a class="reference internal" href="#metadata-custom-entities"><span class="std std-ref">Hierarchical key-value representation of the entity</span></a></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">app-id</span></code></td>
<td>Name of the application</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">program-type</span></code></td>
<td>One of <code class="docutils literal notranslate"><span class="pre">flows</span></code>, <code class="docutils literal notranslate"><span class="pre">mapreduce</span></code>, <code class="docutils literal notranslate"><span class="pre">spark</span></code>, <code class="docutils literal notranslate"><span class="pre">workflows</span></code>, <code class="docutils literal notranslate"><span class="pre">services</span></code>, or <code class="docutils literal notranslate"><span class="pre">workers</span></code></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">program-id</span></code></td>
<td>Name of the program</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">artifact-id</span></code></td>
<td>Name of the artifact</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">artifact-version</span></code></td>
<td>Version of the artifact</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">stream-id</span></code></td>
<td>Name of the stream</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">view-id</span></code></td>
<td>Name of the stream view</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">dataset-id</span></code></td>
<td>Name of the dataset</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">field-name</span></code></td>
<td>Name of the field</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">scope</span></code></td>
<td>Optional scope filter. If not specified, properties in the <code class="docutils literal notranslate"><span class="pre">user</span></code> and
<code class="docutils literal notranslate"><span class="pre">system</span></code> scopes are returned. Otherwise, only properties in the specified scope are returned.</td>
</tr>
</tbody>
</table>
<p class="rubric">HTTP Responses</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Status Codes</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">200</span> <span class="pre">OK</span></code></td>
<td>The tags requested were returned as a JSON string in the body of the response which can be empty if there are no tags associated with the entity or entity does not exist</td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="removing-tags">
<h3>Removing Tags<a class="headerlink" href="#removing-tags" title="Permalink to this headline">🔗</a></h3>
<p>To delete all user metadata tags for an application, dataset, or other entities including custom entities, submit an
HTTP DELETE request:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/&lt;entity-details&gt;/metadata/tags</span>
</pre></div>
</div>
<p>or, for all user metadata tags of a particular program of a specific application:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/apps/&lt;app-id&gt;/&lt;program-type&gt;/&lt;program-id&gt;/metadata/tags</span>
</pre></div>
</div>
<p>or, for a particular version of an artifact:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/artifacts/&lt;artifact-id&gt;/versions/&lt;artifact-version&gt;/metadata/tags</span>
</pre></div>
</div>
<p>or, for a particular view of a stream:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/streams/&lt;stream-id&gt;/views/&lt;view-id&gt;/metadata/tags</span>
</pre></div>
</div>
<p>To delete a specific user metadata tag for an application, dataset, or stream, submit
an HTTP DELETE request with the tag:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/&lt;entity-type&gt;/&lt;entity-id&gt;/metadata/tags/&lt;tag&gt;</span>
</pre></div>
</div>
<p>or, for a particular user metadata tag of a program of a specific application:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/apps/&lt;app-id&gt;/&lt;program-type&gt;/&lt;program-id&gt;/metadata/tags/&lt;tag&gt;</span>
</pre></div>
</div>
<p>or, for a particular version of an artifact:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/artifacts/&lt;artifact-id&gt;/versions/&lt;artifact-version&gt;/metadata/tags/&lt;tag&gt;</span>
</pre></div>
</div>
<p>or, for a particular view of a stream:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/streams/&lt;stream-id&gt;/views/&lt;view-id&gt;/metadata/tags/&lt;tag&gt;</span>
</pre></div>
</div>
<p>or, for a custom entity like field of a dataset:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">DELETE /v3/namespaces/&lt;namespace-id&gt;/datasets/&lt;dataset-id&gt;/field/&lt;field-name&gt;/metadata/tags/&lt;tag&gt;</span>
</pre></div>
</div>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">namespace-id</span></code></td>
<td>Namespace ID</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">entity-details</span></code></td>
<td><a class="reference internal" href="#metadata-custom-entities"><span class="std std-ref">Hierarchical key-value representation of the entity</span></a></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">app-id</span></code></td>
<td>Name of the application</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">program-type</span></code></td>
<td>One of <code class="docutils literal notranslate"><span class="pre">flows</span></code>, <code class="docutils literal notranslate"><span class="pre">mapreduce</span></code>, <code class="docutils literal notranslate"><span class="pre">spark</span></code>, <code class="docutils literal notranslate"><span class="pre">workflows</span></code>, <code class="docutils literal notranslate"><span class="pre">services</span></code>, or <code class="docutils literal notranslate"><span class="pre">workers</span></code></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">program-id</span></code></td>
<td>Name of the program</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">artifact-id</span></code></td>
<td>Name of the artifact</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">artifact-version</span></code></td>
<td>Version of the artifact</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">stream-id</span></code></td>
<td>Name of the stream</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">view-id</span></code></td>
<td>Name of the stream view</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">dataset-id</span></code></td>
<td>Name of the dataset</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">field-name</span></code></td>
<td>Name of the field</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">tag</span></code></td>
<td>Metadata tag</td>
</tr>
</tbody>
</table>
<p class="rubric">HTTP Responses</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Status Codes</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">200</span> <span class="pre">OK</span></code></td>
<td>The method was successfully called, and the tags were deleted, or in the case of a
specific tag, was either deleted or the tag was not present, or the entity itself was not present</td>
</tr>
</tbody>
</table>
<p><strong>Note</strong>: When using this API, only tags in the user scope can be deleted.</p>
</div>
</div>
<div class="section" id="searching-for-metadata">
<span id="http-restful-api-metadata-searching"></span><h2>Searching for Metadata<a class="headerlink" href="#searching-for-metadata" title="Permalink to this headline">🔗</a></h2>
<p>CDAP supports searching metadata of entities. To find which applications, datasets, streams, etc. have a particular
metadata property or metadata tag, submit an HTTP GET request:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/metadata/search?query=&lt;term&gt;[&amp;target=&lt;entity-type&gt;&amp;target=&lt;entity-type2&gt;...][&amp;&lt;option&gt;=&lt;option-value&gt;&amp;...]</span>
</pre></div>
</div>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">namespace-id</span></code></td>
<td>Namespace ID</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">query</span></code></td>
<td><a class="reference internal" href="#http-restful-api-metadata-query-terms"><span class="std std-ref">Query term</span></a>, as described below. Query terms are case-insensitive.</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">entity-type</span></code></td>
<td>Restricts the search to either all or specified entity types: <code class="docutils literal notranslate"><span class="pre">all</span></code>, <code class="docutils literal notranslate"><span class="pre">artifact</span></code>, <code class="docutils literal notranslate"><span class="pre">app</span></code>, <code class="docutils literal notranslate"><span class="pre">dataset</span></code>,
<code class="docutils literal notranslate"><span class="pre">program</span></code>, <code class="docutils literal notranslate"><span class="pre">stream</span></code>, <code class="docutils literal notranslate"><span class="pre">view</span></code></td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">option</span></code></td>
<td><p class="first">Options for controlling cursors, limits, offsets, the inclusion of hidden and custom entities, and sorting:</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Option Name</th>
<th class="head">Option Value, Description, and Notes</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">sort</span></code></td>
<td>The sorting order for the <code class="docutils literal notranslate"><span class="pre">results</span></code> being returned. Default is to sort search results as a function of
relative weights for the specified search query. Specify the sort order as the field name followed by the
sort order (either <code class="docutils literal notranslate"><span class="pre">asc</span></code> or <code class="docutils literal notranslate"><span class="pre">desc</span></code>) with a space separating the two. Using URL-encoding, an example:
<code class="docutils literal notranslate"><span class="pre">&amp;sort=creation-time+asc</span></code>. Note that this field is only applicable when the search query is <code class="docutils literal notranslate"><span class="pre">*</span></code>.</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">offset</span></code></td>
<td>The number of search results to skip before including them in the returned <code class="docutils literal notranslate"><span class="pre">results</span></code>. Default is <code class="docutils literal notranslate"><span class="pre">0</span></code>.</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">limit</span></code></td>
<td>The number of metadata search entities to return in the <code class="docutils literal notranslate"><span class="pre">results</span></code>. By default, there is no limit.</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">cursor</span></code></td>
<td>Cursor to move to in the search results. This would be a value returned in the <code class="docutils literal notranslate"><span class="pre">cursors</span></code> field of a
response of a previous metadata search request. Note that this field is only applicable when the search
query is <code class="docutils literal notranslate"><span class="pre">*</span></code>.</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">numCursors</span></code></td>
<td>Determines the number of chunks of search results of size <code class="docutils literal notranslate"><span class="pre">limit</span></code> to fetch after the first chunk of
size <code class="docutils literal notranslate"><span class="pre">limit</span></code>. This parameter can be used to roughly estimate the total number of results that match
the search query. Only used when the search query is <code class="docutils literal notranslate"><span class="pre">*</span></code>.</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">showHidden</span></code></td>
<td>By default, metadata search hides entities whose name starts with an <code class="docutils literal notranslate"><span class="pre">_</span></code> (underscore) from the search
results. Set this to <code class="docutils literal notranslate"><span class="pre">true</span></code> to include these hidden entities in search results. Default is <code class="docutils literal notranslate"><span class="pre">false</span></code>.</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">showCustom</span></code></td>
<td>By default, metadata search hides custom entities from the search results for backward compatibility.
Set this to <code class="docutils literal notranslate"><span class="pre">true</span></code> to include these custom entities in search results. Default is <code class="docutils literal notranslate"><span class="pre">false</span></code>.</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">entityScope</span></code></td>
<td>The scope of entities for the metadata search. By default, all entities will be returned. Set this to
<code class="docutils literal notranslate"><span class="pre">USER</span></code> to include only user entities; set this to <code class="docutils literal notranslate"><span class="pre">SYSTEM</span></code> to include only system entities.</td>
</tr>
</tbody>
</table>
<p class="last">Format for an option: <code class="docutils literal notranslate"><span class="pre">&amp;&lt;option-name&gt;=&lt;option-value&gt;</span></code></p>
</td>
</tr>
</tbody>
</table>
<p>Entities that match the specified query and entity type are returned in the body of the response in JSON format:</p>
<div class="highlight-json-ellipsis notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
  <span class="nt">&quot;cursors&quot;</span><span class="p">:</span> <span class="p">[</span> <span class="p">],</span>
  <span class="nt">&quot;limit&quot;</span><span class="p">:</span> <span class="mi">20</span><span class="p">,</span>
  <span class="nt">&quot;numCursors&quot;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
  <span class="nt">&quot;offset&quot;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
  <span class="nt">&quot;showHidden&quot;</span><span class="p">:</span> <span class="kc">false</span><span class="p">,</span>
  <span class="nt">&quot;sort&quot;</span><span class="p">:</span> <span class="s2">&quot;creation-time DESC&quot;</span><span class="p">,</span>
  <span class="nt">&quot;total&quot;</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span>
  <span class="nt">&quot;entityScope&quot;</span><span class="p">:</span> <span class="p">[</span> <span class="s2">&quot;SYSTEM&quot;</span> <span class="p">]</span>
  <span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="p">[</span>
      <span class="p">{</span>
          <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
              <span class="nt">&quot;application&quot;</span><span class="p">:</span> <span class="s2">&quot;WordCount&quot;</span><span class="p">,</span>
              <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;PROGRAM&quot;</span><span class="p">,</span>
              <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span><span class="p">,</span>
              <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;RetrieveCounts&quot;</span><span class="p">,</span>
              <span class="nt">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;Service&quot;</span><span class="p">,</span>
              <span class="nt">&quot;version&quot;</span><span class="p">:</span> <span class="s2">&quot;-SNAPSHOT&quot;</span>
          <span class="p">},</span>
          <span class="nt">&quot;metadata&quot;</span><span class="p">:</span> <span class="p">{</span>
              <span class="nt">&quot;SYSTEM&quot;</span><span class="p">:</span> <span class="p">{</span>
                  <span class="nt">&quot;properties&quot;</span><span class="p">:</span> <span class="p">{</span>
                      <span class="nt">&quot;creation-time&quot;</span><span class="p">:</span> <span class="s2">&quot;1482091087438&quot;</span><span class="p">,</span>
                      <span class="nt">&quot;description&quot;</span><span class="p">:</span> <span class="s2">&quot;A service to retrieve statistics, word counts, and associations.&quot;</span><span class="p">,</span>
                      <span class="nt">&quot;entity-name&quot;</span><span class="p">:</span> <span class="s2">&quot;RetrieveCounts&quot;</span><span class="p">,</span>
                      <span class="nt">&quot;version&quot;</span><span class="p">:</span> <span class="s2">&quot;-SNAPSHOT&quot;</span>
                  <span class="p">},</span>
                  <span class="nt">&quot;tags&quot;</span><span class="p">:</span> <span class="p">[</span>
                      <span class="s2">&quot;Realtime&quot;</span><span class="p">,</span>
                      <span class="s2">&quot;Service&quot;</span>
                  <span class="p">]</span>
              <span class="p">}</span>
          <span class="p">}</span>
      <span class="p">},</span>
      <span class="p">{</span>
          <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
              <span class="nt">&quot;application&quot;</span><span class="p">:</span> <span class="s2">&quot;WordCount&quot;</span><span class="p">,</span>
              <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;PROGRAM&quot;</span><span class="p">,</span>
              <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span><span class="p">,</span>
              <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;WordCounter&quot;</span><span class="p">,</span>
              <span class="nt">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;Flow&quot;</span><span class="p">,</span>
              <span class="nt">&quot;version&quot;</span><span class="p">:</span> <span class="s2">&quot;-SNAPSHOT&quot;</span>
          <span class="p">},</span>
          <span class="nt">&quot;metadata&quot;</span><span class="p">:</span> <span class="p">{</span>
              <span class="nt">&quot;SYSTEM&quot;</span><span class="p">:</span> <span class="p">{</span>
                  <span class="nt">&quot;properties&quot;</span><span class="p">:</span> <span class="p">{</span>
                      <span class="nt">&quot;creation-time&quot;</span><span class="p">:</span> <span class="s2">&quot;1482091087390&quot;</span><span class="p">,</span>
                      <span class="nt">&quot;description&quot;</span><span class="p">:</span> <span class="s2">&quot;Example Word Count Flow&quot;</span><span class="p">,</span>
                      <span class="nt">&quot;entity-name&quot;</span><span class="p">:</span> <span class="s2">&quot;WordCounter&quot;</span><span class="p">,</span>
                      <span class="nt">&quot;version&quot;</span><span class="p">:</span> <span class="s2">&quot;-SNAPSHOT&quot;</span>
                  <span class="p">},</span>
                  <span class="nt">&quot;tags&quot;</span><span class="p">:</span> <span class="p">[</span>
                      <span class="s2">&quot;Flow&quot;</span><span class="p">,</span>
                      <span class="s2">&quot;Realtime&quot;</span>
                  <span class="p">]</span>
              <span class="p">}</span>
          <span class="p">}</span>
      <span class="p">}</span>
  <span class="p">]</span>
<span class="p">}</span>
</pre></div>
</div>
<p class="rubric">HTTP Responses</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Status Codes</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">200</span> <span class="pre">OK</span></code></td>
<td>Entity ID and metadata of entities that match the query and entity type(s) are returned in the body of the response</td>
</tr>
</tbody>
</table>
<p class="rubric" id="http-restful-api-metadata-query-terms">Query Terms</p>
<p>CDAP supports prefix-based search of metadata properties and tags across both <em>user</em> and <em>system</em> scopes.
Search metadata of entities by using either a complete or partial name followed by an asterisk <code class="docutils literal notranslate"><span class="pre">*</span></code>.</p>
<p>Search for properties and tags by specifying one of:</p>
<ul class="simple">
<li>a complete property key-value pair, separated by a colon, such as <code class="docutils literal notranslate"><span class="pre">type:production</span></code></li>
<li>a complete property key with a partial value, such as <code class="docutils literal notranslate"><span class="pre">type:prod*</span></code></li>
<li>a complete <code class="docutils literal notranslate"><span class="pre">tags</span></code> key with a complete or partial value, such as <code class="docutils literal notranslate"><span class="pre">tags:production</span></code> or <code class="docutils literal notranslate"><span class="pre">tags:prod*</span></code>
to search for tags only</li>
<li>a complete or partial value, such as <code class="docutils literal notranslate"><span class="pre">prod*</span></code>; this will return both properties and tags</li>
<li>multiple search terms separated by space, such as <code class="docutils literal notranslate"><span class="pre">type:prod*</span> <span class="pre">author:joe</span></code>; this will return entities having
either of the terms in their metadata.</li>
</ul>
<p>Since CDAP also annotates <em>system</em> metadata to entities by default as mentioned at
<a class="reference external" href="../../../developer-manual/metadata/system-metadata.html#metadata-system-metadata" title="(in Cask Data Application Platform v6.1.1)"><span class="xref std std-ref">System Metadata</span></a>, the following <em>special</em> search queries are also supported:</p>
<ul class="simple">
<li>artifacts or applications containing a specific plugin: <code class="docutils literal notranslate"><span class="pre">plugin:&lt;plugin-name&gt;</span></code></li>
<li>programs with a specific mode: <code class="docutils literal notranslate"><span class="pre">batch</span></code> or <code class="docutils literal notranslate"><span class="pre">realtime</span></code></li>
<li>applications with a specific program type: <code class="docutils literal notranslate"><span class="pre">flow:&lt;flow-name&gt;</span></code>, <code class="docutils literal notranslate"><span class="pre">service:&lt;service-name&gt;</span></code>,
<code class="docutils literal notranslate"><span class="pre">mapreduce:&lt;mapreduce-name&gt;</span></code>, <code class="docutils literal notranslate"><span class="pre">spark:&lt;spark-name&gt;</span></code>, <code class="docutils literal notranslate"><span class="pre">worker:&lt;worker-name&gt;</span></code>,
<code class="docutils literal notranslate"><span class="pre">workflow:&lt;workflow-name&gt;</span></code></li>
<li>datasets, streams or views with schema field:<ul>
<li>field name only: <code class="docutils literal notranslate"><span class="pre">field-name</span></code></li>
<li>field name with a type: <code class="docutils literal notranslate"><span class="pre">&lt;field-name&gt;:&lt;field-type&gt;</span></code>, where <code class="docutils literal notranslate"><span class="pre">field-type</span></code> can be:<ul>
<li>simple types: <code class="docutils literal notranslate"><span class="pre">int</span></code>, <code class="docutils literal notranslate"><span class="pre">long</span></code>, <code class="docutils literal notranslate"><span class="pre">boolean</span></code>, <code class="docutils literal notranslate"><span class="pre">float</span></code>, <code class="docutils literal notranslate"><span class="pre">double</span></code>, <code class="docutils literal notranslate"><span class="pre">bytes</span></code>, <code class="docutils literal notranslate"><span class="pre">string</span></code>, <code class="docutils literal notranslate"><span class="pre">enum</span></code></li>
<li>complex types: <code class="docutils literal notranslate"><span class="pre">array</span></code>, <code class="docutils literal notranslate"><span class="pre">map</span></code>, <code class="docutils literal notranslate"><span class="pre">record</span></code>, <code class="docutils literal notranslate"><span class="pre">union</span></code></li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="highlight-json-ellipsis notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
   <span class="nt">&quot;type&quot;</span><span class="p">:</span><span class="s2">&quot;record&quot;</span><span class="p">,</span>
   <span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="s2">&quot;employee&quot;</span><span class="p">,</span>
   <span class="nt">&quot;fields&quot;</span><span class="p">:[</span>
      <span class="p">{</span>
         <span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="s2">&quot;employeeName&quot;</span><span class="p">,</span>
         <span class="nt">&quot;type&quot;</span><span class="p">:</span><span class="s2">&quot;string&quot;</span>
      <span class="p">},</span>
      <span class="p">{</span>
         <span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="s2">&quot;departments&quot;</span><span class="p">,</span>
         <span class="nt">&quot;type&quot;</span><span class="p">:{</span>
            <span class="nt">&quot;type&quot;</span><span class="p">:</span><span class="s2">&quot;array&quot;</span><span class="p">,</span>
            <span class="nt">&quot;items&quot;</span><span class="p">:</span><span class="s2">&quot;long&quot;</span>
         <span class="p">}</span>
      <span class="p">}</span>
   <span class="p">]</span>
<span class="p">}</span>
</pre></div>
</div>
<p>With a schema as shown above, queries such as <code class="docutils literal notranslate"><span class="pre">employee:record</span></code>, <code class="docutils literal notranslate"><span class="pre">employeeName:string</span></code>, <code class="docutils literal notranslate"><span class="pre">departments</span></code>,
<code class="docutils literal notranslate"><span class="pre">departments:array</span></code> can be issued.</p>
</div>
<div class="section" id="viewing-lineages">
<span id="http-restful-api-metadata-lineage"></span><h2>Viewing Lineages<a class="headerlink" href="#viewing-lineages" title="Permalink to this headline">🔗</a></h2>
<p>To view the lineage of a dataset or stream, submit an HTTP GET request:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/&lt;entity-type&gt;/&lt;entity-id&gt;/lineage?start=&lt;start-ts&gt;&amp;end=&lt;end-ts&gt;[&amp;levels=&lt;levels&gt;][&amp;collapse=&lt;collapse&gt;&amp;collapse=&lt;collapse&gt;...]</span>
</pre></div>
</div>
<p>where:</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">namespace-id</span></code></td>
<td>Namespace ID</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">entity-type</span></code></td>
<td>One of <code class="docutils literal notranslate"><span class="pre">datasets</span></code> or <code class="docutils literal notranslate"><span class="pre">streams</span></code></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">entity-id</span></code></td>
<td>Name of the <code class="docutils literal notranslate"><span class="pre">dataset</span></code> or <code class="docutils literal notranslate"><span class="pre">stream</span></code></td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">start-ts</span></code></td>
<td>Starting time-stamp of lineage (inclusive), in seconds. Supports <code class="docutils literal notranslate"><span class="pre">now</span></code>, <code class="docutils literal notranslate"><span class="pre">now-1h</span></code>, etc. syntax.</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">end-ts</span></code></td>
<td>Ending time-stamp of lineage (exclusive), in seconds. Supports <code class="docutils literal notranslate"><span class="pre">now</span></code>, <code class="docutils literal notranslate"><span class="pre">now-1h</span></code>, etc. syntax.</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">levels</span></code></td>
<td>Number of levels of lineage output to return. Defaults to 10. Determines how far back the provenance
of the data in the lineage chain is calculated.</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">collapse</span></code></td>
<td>An optional set of <code class="docutils literal notranslate"><span class="pre">collapse</span></code> types (any of <code class="docutils literal notranslate"><span class="pre">access</span></code>, <code class="docutils literal notranslate"><span class="pre">run</span></code>, or <code class="docutils literal notranslate"><span class="pre">component</span></code>) by which to
<a class="reference internal" href="#http-restful-api-metadata-lineage-collapse"><span class="std std-ref">collapse the lineage output</span></a>.
By default, lineage output is not collapsed. Multiple collapse parameters are supported.</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">rollup</span></code></td>
<td>An optional <code class="docutils literal notranslate"><span class="pre">rollup</span></code> type to use to <a class="reference internal" href="#http-restful-api-metadata-lineage-rollup"><span class="std std-ref">rollup the lineage output</span></a>. By default, lineage output is not rolled up.
Currently supports the value <code class="docutils literal notranslate"><span class="pre">workflow</span></code>.</td>
</tr>
</tbody>
</table>
<p>See in the Metrics HTTP RESTful API <a class="reference internal" href="metrics.html#http-restful-api-metrics-time-range"><span class="std std-ref">Querying by a Time Range</span></a>
for examples of the “now” time syntax.</p>
<p>For more information about collapsing lineage output, please refer to the section below on
<a class="reference internal" href="#http-restful-api-metadata-lineage-collapse"><span class="std std-ref">Collapsing Lineage Output</span></a>.</p>
<p>The lineage will be returned as a JSON string in the body of the response. The JSON describes lineage as a graph
of connections between programs and datasets (or streams) in the specified time range. The number of
levels of the request (<code class="docutils literal notranslate"><span class="pre">levels</span></code>) determines the depth of the graph. This impacts how far back the provenance of the
data in the lineage chain is calculated, as described in the <a class="reference external" href="../../../developer-manual/metadata/discovery-lineage.html#metadata-lineage" title="(in Cask Data Application Platform v6.1.1)"><span class="xref std std-ref">Metadata and Lineage</span></a>.</p>
<p>Lineage JSON consists of three main sections:</p>
<ul class="simple">
<li><strong>Relations:</strong> contains information on data accessed by programs.
Access type can be <em>read</em>, <em>write</em>, <em>both</em>, or <em>unknown</em>.
It also contains the <em>runid</em> of the program that accessed the data,
and the specifics of any <em>component</em> of a program
that also accessed the data. For example, a flowlet is a <em>component</em> of a flow.</li>
<li><strong>Data:</strong> contains Datasets or Streams that were accessed by programs.</li>
<li><strong>Programs:</strong> contains information on programs (flows, MapReduce, Spark, workers, etc.)
that accessed the data.</li>
</ul>
<p>Here is an example, pretty-printed:</p>
<div class="highlight-json-ellipsis notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
    <span class="nt">&quot;start&quot;</span><span class="p">:</span> <span class="mi">1442863938</span><span class="p">,</span>
    <span class="nt">&quot;end&quot;</span><span class="p">:</span> <span class="mi">1442881938</span><span class="p">,</span>
    <span class="nt">&quot;relations&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="p">{</span>
            <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;stream.default.purchaseStream&quot;</span><span class="p">,</span>
            <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;flows.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
            <span class="nt">&quot;access&quot;</span><span class="p">:</span> <span class="s2">&quot;read&quot;</span><span class="p">,</span>
            <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
                <span class="s2">&quot;4b5d7891-60a7-11e5-a9b0-42010af01c4d&quot;</span>
            <span class="p">],</span>
            <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
                <span class="s2">&quot;reader&quot;</span>
            <span class="p">]</span>
        <span class="p">},</span>
        <span class="p">{</span>
            <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
            <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;flows.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
            <span class="nt">&quot;access&quot;</span><span class="p">:</span> <span class="s2">&quot;unknown&quot;</span><span class="p">,</span>
            <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
                <span class="s2">&quot;4b5d7891-60a7-11e5-a9b0-42010af01c4d&quot;</span>
            <span class="p">],</span>
            <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
                <span class="s2">&quot;collector&quot;</span>
            <span class="p">]</span>
        <span class="p">}</span>
    <span class="p">],</span>
    <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="p">{</span>
        <span class="nt">&quot;dataset.default.purchases&quot;</span><span class="p">:</span> <span class="p">{</span>
            <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
                <span class="nt">&quot;dataset&quot;</span><span class="p">:</span> <span class="s2">&quot;history&quot;</span><span class="p">,</span>
                <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;DATASET&quot;</span><span class="p">,</span>
                <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span>
            <span class="p">}</span>
        <span class="p">},</span>
        <span class="nt">&quot;stream.default.purchaseStream&quot;</span><span class="p">:</span> <span class="p">{</span>
            <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
                <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;STREAM&quot;</span><span class="p">,</span>
                <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span><span class="p">,</span>
                <span class="nt">&quot;stream&quot;</span><span class="p">:</span> <span class="s2">&quot;purchaseStream&quot;</span>
            <span class="p">}</span>
        <span class="p">}</span>
    <span class="p">},</span>
    <span class="nt">&quot;programs&quot;</span><span class="p">:</span> <span class="p">{</span>
        <span class="nt">&quot;flows.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">:</span> <span class="p">{</span>
            <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
                <span class="nt">&quot;application&quot;</span><span class="p">:</span> <span class="s2">&quot;PurchaseHistory&quot;</span><span class="p">,</span>
                <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;PROGRAM&quot;</span><span class="p">,</span>
                <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span><span class="p">,</span>
                <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;PurchaseFlow&quot;</span><span class="p">,</span>
                <span class="nt">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;Flow&quot;</span><span class="p">,</span>
                <span class="nt">&quot;version&quot;</span><span class="p">:</span> <span class="s2">&quot;-SNAPSHOT&quot;</span>
            <span class="p">}</span>
        <span class="p">}</span>
    <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p class="rubric">HTTP Responses</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Status Codes</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">200</span> <span class="pre">OK</span></code></td>
<td>Entities IDs of entities with the metadata properties specified were returned as a
list of strings in the body of the response</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">404</span> <span class="pre">NOT</span> <span class="pre">FOUND</span></code></td>
<td>No entities matching the specified query were found</td>
</tr>
</tbody>
</table>
<div class="section" id="collapsing-lineage-output">
<span id="http-restful-api-metadata-lineage-collapse"></span><h3>Collapsing Lineage Output<a class="headerlink" href="#collapsing-lineage-output" title="Permalink to this headline">🔗</a></h3>
<p>Lineage output can be collapsed by <code class="docutils literal notranslate"><span class="pre">access</span></code>, <code class="docutils literal notranslate"><span class="pre">run</span></code>, or <code class="docutils literal notranslate"><span class="pre">component</span></code>. Collapsing allows you to group all the lineage
relations for the specified collapse-type together in the lineage output. Collapsing is useful when you do not want
to differentiate between multiple lineage relations that only differ by the collapse-type.</p>
<p>For example, consider a program that wrote to a dataset in multiple runs over a specified time interval.
If you do not want to differentiate between lineage relations involving different runs of this program
(so you only want to know that a program accessed a data entity in a given time interval), you could provide
the query parameter <code class="docutils literal notranslate"><span class="pre">collapse=run</span></code> to the lineage API. This would collapse the lineage relations in the
output to group the multiple runs of the program together.</p>
<p>You can also collapse lineage relations by <cite>access</cite> (which will group together those relations that
differ only by access type) or by <cite>component</cite> (which will group together those that differ only by
component together). The lineage HTTP RESTful API also allows you to use multiple <cite>collapse`</cite> parameters
in the same query.</p>
<p>Consider these relations from the output of a lineage API request:</p>
<div class="highlight-json-ellipsis notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
  <span class="nt">&quot;relations&quot;</span><span class="p">:</span> <span class="p">[</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;read&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;a442db61-0c2f-11e6-bc75-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;read&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;reader&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;a442db61-0c2f-11e6-bc75-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;read&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;ae188ea2-0c2f-11e6-b499-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;write&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;a442db61-0c2f-11e6-bc75-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;write&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchase&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;ae188ea2-0c2f-11e6-b499-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">}</span>
  <span class="p">]</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Collapsing the above by <code class="docutils literal notranslate"><span class="pre">run</span></code> would group the runs together as:</p>
<div class="highlight-json-ellipsis notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
  <span class="nt">&quot;relations&quot;</span><span class="p">:</span> <span class="p">[</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;read&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;a442db61-0c2f-11e6-bc75-561602fdb525&quot;</span><span class="p">,</span>
        <span class="s2">&quot;ae188ea2-0c2f-11e6-b499-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;read&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;reader&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;a442db61-0c2f-11e6-bc75-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;write&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;a442db61-0c2f-11e6-bc75-561602fdb525&quot;</span><span class="p">,</span>
        <span class="s2">&quot;ae188ea2-0c2f-11e6-b499-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">}</span>
  <span class="p">]</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Collapsing by <code class="docutils literal notranslate"><span class="pre">access</span></code> would produce:</p>
<div class="highlight-json-ellipsis notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
  <span class="nt">&quot;relations&quot;</span><span class="p">:</span> <span class="p">[</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;read&quot;</span><span class="p">,</span>
        <span class="s2">&quot;write&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;a442db61-0c2f-11e6-bc75-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;read&quot;</span><span class="p">,</span>
        <span class="s2">&quot;write&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;ae188ea2-0c2f-11e6-b499-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;read&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;reader&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;a442db61-0c2f-11e6-bc75-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">}</span>
  <span class="p">]</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Similarly, collapsing by <code class="docutils literal notranslate"><span class="pre">component</span></code> will generate:</p>
<div class="highlight-json-ellipsis notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
  <span class="nt">&quot;relations&quot;</span><span class="p">:</span> <span class="p">[</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;read&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span><span class="p">,</span>
        <span class="s2">&quot;reader&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;a442db61-0c2f-11e6-bc75-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;read&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;ae188ea2-0c2f-11e6-b499-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;write&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;a442db61-0c2f-11e6-bc75-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
      <span class="nt">&quot;accesses&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;write&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchase&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.PurchaseFlow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;ae188ea2-0c2f-11e6-b499-561602fdb525&quot;</span>
      <span class="p">]</span>
    <span class="p">}</span>
  <span class="p">]</span>
<span class="p">}</span>
</pre></div>
</div>
</div>
<div class="section" id="rolling-up-lineage-output">
<span id="http-restful-api-metadata-lineage-rollup"></span><h3>Rolling Up Lineage Output<a class="headerlink" href="#rolling-up-lineage-output" title="Permalink to this headline">🔗</a></h3>
<p>Lineage rollup allows you to group multiple entities together for a condensed view in the lineage output.</p>
<p>Currently, for the parameter <code class="docutils literal notranslate"><span class="pre">rollup</span></code>, only the value <code class="docutils literal notranslate"><span class="pre">workflow</span></code> is supported, which allows programs
to be grouped together into workflows if multiple programs are created as part of a workflow.</p>
<p>For example, suppose you have a workflow that starts two programs to complete an associated task. If you do not want to
differentiate between lineage relations involving different programs of this workflow, you could provide the query
parameter <code class="docutils literal notranslate"><span class="pre">rollup=workflow</span></code> to the lineage API. This would rollup the lineage relations in the output to show
corresponding workflows instead of individual programs.</p>
<p>Consider these relations from the output of a lineage API request:</p>
<div class="highlight-json-ellipsis notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
  <span class="nt">&quot;start&quot;</span><span class="p">:</span> <span class="mi">1442863938</span><span class="p">,</span>
  <span class="nt">&quot;end&quot;</span><span class="p">:</span> <span class="mi">1442881938</span><span class="p">,</span>
  <span class="nt">&quot;relations&quot;</span><span class="p">:</span> <span class="p">[</span>
    <span class="p">{</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.phase-1&quot;</span><span class="p">,</span>
      <span class="nt">&quot;access&quot;</span><span class="p">:</span> <span class="s2">&quot;read&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;4b5d7891-60a7-11e5-a9b0-42010af01c4d&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;reader&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;mapreduce.default.PurchaseHistory.phase-2&quot;</span><span class="p">,</span>
      <span class="nt">&quot;access&quot;</span><span class="p">:</span> <span class="s2">&quot;unknown&quot;</span><span class="p">,</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;7d6r7891-60a7-11e5-a9b0-42010af01c4d&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;collector&quot;</span>
      <span class="p">]</span>
    <span class="p">}</span>
  <span class="p">],</span>
  <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="p">{</span>
    <span class="nt">&quot;dataset.default.purchases&quot;</span><span class="p">:</span> <span class="p">{</span>
      <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
          <span class="nt">&quot;dataset&quot;</span><span class="p">:</span> <span class="s2">&quot;purchases&quot;</span><span class="p">,</span>
          <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;DATASET&quot;</span><span class="p">,</span>
          <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span>
      <span class="p">}</span>
    <span class="p">},</span>
  <span class="p">},</span>
  <span class="nt">&quot;programs&quot;</span><span class="p">:</span> <span class="p">{</span>
    <span class="nt">&quot;mapreduce.default.PurchaseHistory.phase-1&quot;</span><span class="p">:</span> <span class="p">{</span>
      <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
          <span class="nt">&quot;application&quot;</span><span class="p">:</span> <span class="s2">&quot;PurchaseHistory&quot;</span><span class="p">,</span>
          <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;PROGRAM&quot;</span><span class="p">,</span>
          <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span><span class="p">,</span>
          <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;phase-1&quot;</span><span class="p">,</span>
          <span class="nt">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;Mapreduce&quot;</span><span class="p">,</span>
          <span class="nt">&quot;version&quot;</span><span class="p">:</span> <span class="s2">&quot;-SNAPSHOT&quot;</span>
      <span class="p">}</span>
    <span class="p">},</span>
    <span class="nt">&quot;mapreduce.default.PurchaseHistory.phase-2&quot;</span><span class="p">:</span> <span class="p">{</span>
      <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
          <span class="nt">&quot;application&quot;</span><span class="p">:</span> <span class="s2">&quot;PurchaseHistory&quot;</span><span class="p">,</span>
          <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;PROGRAM&quot;</span><span class="p">,</span>
          <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span><span class="p">,</span>
          <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;phase-2&quot;</span><span class="p">,</span>
          <span class="nt">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;Mapreduce&quot;</span><span class="p">,</span>
          <span class="nt">&quot;version&quot;</span><span class="p">:</span> <span class="s2">&quot;-SNAPSHOT&quot;</span>
      <span class="p">}</span>
    <span class="p">}</span>
  <span class="p">},</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Rolling up the above using <code class="docutils literal notranslate"><span class="pre">rollup=workflow</span></code> would group the programs together as:</p>
<div class="highlight-json-ellipsis notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
  <span class="nt">&quot;start&quot;</span><span class="p">:</span> <span class="mi">1442863938</span><span class="p">,</span>
  <span class="nt">&quot;end&quot;</span><span class="p">:</span> <span class="mi">1442881938</span><span class="p">,</span>
  <span class="nt">&quot;relations&quot;</span><span class="p">:</span> <span class="p">[</span>
    <span class="p">{</span>
      <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="s2">&quot;dataset.default.purchases&quot;</span><span class="p">,</span>
      <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;workflows.default.PurchaseHistory.DataPipelineWorkflow&quot;</span><span class="p">,</span>
      <span class="nt">&quot;access&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;read&quot;</span><span class="p">,</span>
        <span class="s2">&quot;unknown&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;runs&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;5b3ar7891-60a7-11e5-a9b0-42010af01c4d&quot;</span>
      <span class="p">],</span>
      <span class="nt">&quot;components&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;reader&quot;</span>
      <span class="p">]</span>
    <span class="p">},</span>
  <span class="p">],</span>
  <span class="nt">&quot;data&quot;</span><span class="p">:</span> <span class="p">{</span>
    <span class="nt">&quot;dataset.default.purchases&quot;</span><span class="p">:</span> <span class="p">{</span>
      <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
        <span class="nt">&quot;dataset&quot;</span><span class="p">:</span> <span class="s2">&quot;purchases&quot;</span><span class="p">,</span>
        <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;DATASET&quot;</span><span class="p">,</span>
        <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span>
      <span class="p">}</span>
    <span class="p">},</span>
  <span class="p">},</span>
  <span class="nt">&quot;programs&quot;</span><span class="p">:</span> <span class="p">{</span>
    <span class="nt">&quot;workflows.default.PurchaseHistory.DataPipelineWorkflow&quot;</span><span class="p">:</span> <span class="p">{</span>
      <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
          <span class="nt">&quot;application&quot;</span><span class="p">:</span> <span class="s2">&quot;PurchaseHistory&quot;</span><span class="p">,</span>
          <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;PROGRAM&quot;</span><span class="p">,</span>
          <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span><span class="p">,</span>
          <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;DataPipelineWorkflow&quot;</span><span class="p">,</span>
          <span class="nt">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;Workflow&quot;</span><span class="p">,</span>
          <span class="nt">&quot;version&quot;</span><span class="p">:</span> <span class="s2">&quot;-SNAPSHOT&quot;</span>
      <span class="p">}</span>
    <span class="p">},</span>
  <span class="p">},</span>
<span class="p">}</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="field-level-lineage">
<span id="http-restful-api-metadata-fieldlevellineage"></span><h2>Field Level Lineage<a class="headerlink" href="#field-level-lineage" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="fields-associated-with-the-dataset">
<h3>Fields associated with the Dataset<a class="headerlink" href="#fields-associated-with-the-dataset" title="Permalink to this headline">🔗</a></h3>
<p>Gets the fields that were associated with the dataset for the specified range of time:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /namespaces/{namespace-id}/datasets/{dataset-id}/lineage/fields?start=&lt;start-ts&gt;&amp;end=&lt;end-ts&gt;[&amp;prefix=&lt;prefix&gt;]</span>
</pre></div>
</div>
<p>where:</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">namespace-id</span></code></td>
<td>Namespace ID</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">dataset-id</span></code></td>
<td>Name of the <code class="docutils literal notranslate"><span class="pre">dataset</span></code></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">start-ts</span></code></td>
<td>Starting time-stamp (inclusive), in seconds. Supports <code class="docutils literal notranslate"><span class="pre">now</span></code>, <code class="docutils literal notranslate"><span class="pre">now-1h</span></code>, etc. syntax.</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">end-ts</span></code></td>
<td>Ending time-stamp (exclusive), in seconds. Supports <code class="docutils literal notranslate"><span class="pre">now</span></code>, <code class="docutils literal notranslate"><span class="pre">now-1h</span></code>, etc. syntax.</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">prefix</span></code></td>
<td>Optional <code class="docutils literal notranslate"><span class="pre">prefix</span></code>, when provided only fields that have given prefix will be returned</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">includeCurrent</span></code></td>
<td>Optional flag, when set to true the current fields of the dataset will be be included irrespective of whether they have any lineage information or not.</td>
</tr>
</tbody>
</table>
<p>Following is sample response:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">[{&quot;name&quot;:&quot;firstName&quot;,&quot;lineage&quot;:true},{&quot;name&quot;:&quot;lastName&quot;,&quot;lineage&quot;:true},{&quot;name&quot;:&quot;customer_id&quot;,&quot;lineage&quot;:false}]</span>
</pre></div>
</div>
<p class="rubric">HTTP Responses</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Status Codes</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">200</span> <span class="pre">OK</span></code></td>
<td>Fields of dataset are returned as a list of strings in the body of the response</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">400</span> <span class="pre">BAD</span> <span class="pre">REQUEST</span></code></td>
<td>Failure to parse the time range provided</td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="field-lineage-summary">
<h3>Field Lineage Summary<a class="headerlink" href="#field-lineage-summary" title="Permalink to this headline">🔗</a></h3>
<p>Gets the field lineage summary for a specified field of a dataset. The field lineage summary consists of the
sets of datasets and their respective fields used to compute the specified field of a dataset:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /namespaces/{namespace-id}/datasets/{dataset-id}/lineage/fields/{field-name}?start=&lt;start-ts&gt;&amp;end=&lt;end-ts&gt;&amp;direction=incoming</span>
</pre></div>
</div>
<p>where:</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">namespace-id</span></code></td>
<td>Namespace ID</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">dataset-id</span></code></td>
<td>Name of the <code class="docutils literal notranslate"><span class="pre">dataset</span></code></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">field-name</span></code></td>
<td>Name of the <code class="docutils literal notranslate"><span class="pre">field</span></code></td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">start-ts</span></code></td>
<td>Starting time-stamp (inclusive), in seconds. Supports <code class="docutils literal notranslate"><span class="pre">now</span></code>, <code class="docutils literal notranslate"><span class="pre">now-1h</span></code>, etc. syntax.</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">end-ts</span></code></td>
<td>Ending time-stamp (exclusive), in seconds. Supports <code class="docutils literal notranslate"><span class="pre">now</span></code>, <code class="docutils literal notranslate"><span class="pre">now-1h</span></code>, etc. syntax.</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">direction</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">incoming</span></code>, to return the set of dataset and fields which participated in the computation of the given field. <code class="docutils literal notranslate"><span class="pre">outgoing</span></code>, to return the set of dataset and fields to whose computation the given field participated. <code class="docutils literal notranslate"><span class="pre">both</span></code>, to return both incoming and outgoing.</td>
</tr>
</tbody>
</table>
<p>The returned response consists of the direction in which the summary is requested and the datasets and fields
that were responsible for the computation of a specified field. Currently, the only supported direction is <code class="docutils literal notranslate"><span class="pre">incoming</span></code>.</p>
<p>Following is a sample response:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">{</span>
<span class="go">    &quot;incoming&quot;: [</span>
<span class="go">        {</span>
<span class="go">            &quot;dataset&quot;: {</span>
<span class="go">                &quot;dataset&quot;: &quot;Customer&quot;,</span>
<span class="go">                &quot;entity&quot;: &quot;DATASET&quot;,</span>
<span class="go">                &quot;namespace&quot;: &quot;default&quot;</span>
<span class="go">            },</span>
<span class="go">            &quot;fields&quot;: [</span>
<span class="go">                &quot;body&quot;</span>
<span class="go">            ]</span>
<span class="go">        },</span>
<span class="go">        {</span>
<span class="go">            &quot;dataset&quot;: {</span>
<span class="go">                &quot;dataset&quot;: &quot;purchases&quot;,</span>
<span class="go">                &quot;entity&quot;: &quot;DATASET&quot;,</span>
<span class="go">                &quot;namespace&quot;: &quot;default&quot;</span>
<span class="go">            },</span>
<span class="go">            &quot;fields&quot;: [</span>
<span class="go">                &quot;body&quot;</span>
<span class="go">            ]</span>
<span class="go">        }</span>
<span class="go">    ]</span>
<span class="go">}</span>
</pre></div>
</div>
<p class="rubric">HTTP Responses</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Status Codes</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">200</span> <span class="pre">OK</span></code></td>
<td>Fields of dataset are returned as a list of strings in the body of the response</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">400</span> <span class="pre">BAD</span> <span class="pre">REQUEST</span></code></td>
<td>Failure to parse the time range provided</td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="field-lineage-operations">
<h3>Field Lineage Operations<a class="headerlink" href="#field-lineage-operations" title="Permalink to this headline">🔗</a></h3>
<p>Gets the details of operations responsible for computation of a specified field of a dataset for a specified range of time:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /namespaces/{namespace-id}/datasets/{dataset-id}/lineage/fields/{field-name}/operations?start=&lt;start-ts&gt;&amp;end=&lt;end-ts&gt;&amp;direction=incoming</span>
</pre></div>
</div>
<p>where:</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">namespace-id</span></code></td>
<td>Namespace ID</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">dataset-id</span></code></td>
<td>Name of the <code class="docutils literal notranslate"><span class="pre">dataset</span></code></td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">field-name</span></code></td>
<td>Name of the <code class="docutils literal notranslate"><span class="pre">field</span></code></td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">start-ts</span></code></td>
<td>Starting time-stamp (inclusive), in seconds. Supports <code class="docutils literal notranslate"><span class="pre">now</span></code>, <code class="docutils literal notranslate"><span class="pre">now-1h</span></code>, etc. syntax.</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">end-ts</span></code></td>
<td>Ending time-stamp (exclusive), in seconds. Supports <code class="docutils literal notranslate"><span class="pre">now</span></code>, <code class="docutils literal notranslate"><span class="pre">now-1h</span></code>, etc. syntax.</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">direction</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">incoming</span></code>, to return the operations which participated in the computation of the given field. <code class="docutils literal notranslate"><span class="pre">outgoing</span></code>, to return the operations in which this field participated. <code class="docutils literal notranslate"><span class="pre">both</span></code>, to return both incoming and outgoing.</td>
</tr>
</tbody>
</table>
<p>The single field can be computed in multiple ways, where each unique way consists of a list of operations that
participated in the computation and the list of programs that performed the computation. The returned response
consists of the direction in which the operations are requested. Currently, the only supported direction is <code class="docutils literal notranslate"><span class="pre">incoming</span></code>.
For the specified direction, the response includes the different ways that the field was computed.</p>
<p>Following is a sample response:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">{</span>
<span class="go">  &quot;incoming&quot;: [</span>
<span class="go">    {</span>
<span class="go">        &quot;operations&quot;: [</span>
<span class="go">            {</span>
<span class="go">                &quot;description&quot;: &quot;Read files&quot;,</span>
<span class="go">                &quot;inputs&quot;: {</span>
<span class="go">                    &quot;endPoint&quot;: {</span>
<span class="go">                        &quot;name&quot;: &quot;purchases&quot;,</span>
<span class="go">                        &quot;namespace&quot;: &quot;default&quot;</span>
<span class="go">                    }</span>
<span class="go">                },</span>
<span class="go">                &quot;name&quot;: &quot;File2.Read&quot;,</span>
<span class="go">                &quot;outputs&quot;: {</span>
<span class="go">                    &quot;fields&quot;: [</span>
<span class="go">                        &quot;offset&quot;,</span>
<span class="go">                        &quot;body&quot;</span>
<span class="go">                    ]</span>
<span class="go">                }</span>
<span class="go">            },</span>
<span class="go">            {</span>
<span class="go">                &quot;description&quot;: &quot;Parsed field&quot;,</span>
<span class="go">                &quot;inputs&quot;: {</span>
<span class="go">                    &quot;fields&quot;: [</span>
<span class="go">                        {</span>
<span class="go">                            &quot;name&quot;: &quot;body&quot;,</span>
<span class="go">                            &quot;origin&quot;: &quot;File2.Read&quot;</span>
<span class="go">                        }</span>
<span class="go">                    ]</span>
<span class="go">                },</span>
<span class="go">                &quot;name&quot;: &quot;CSVParser2.CSV Parse&quot;,</span>
<span class="go">                &quot;outputs&quot;: {</span>
<span class="go">                    &quot;fields&quot;: [</span>
<span class="go">                        &quot;customer_id&quot;,</span>
<span class="go">                        &quot;item&quot;,</span>
<span class="go">                        &quot;price&quot;</span>
<span class="go">                    ]</span>
<span class="go">                }</span>
<span class="go">            },</span>
<span class="go">            {</span>
<span class="go">                &quot;description&quot;: &quot;Read files&quot;,</span>
<span class="go">                &quot;inputs&quot;: {</span>
<span class="go">                    &quot;endPoint&quot;: {</span>
<span class="go">                        &quot;name&quot;: &quot;Customer&quot;,</span>
<span class="go">                        &quot;namespace&quot;: &quot;default&quot;</span>
<span class="go">                    }</span>
<span class="go">                },</span>
<span class="go">                &quot;name&quot;: &quot;File.Read&quot;,</span>
<span class="go">                &quot;outputs&quot;: {</span>
<span class="go">                    &quot;fields&quot;: [</span>
<span class="go">                        &quot;offset&quot;,</span>
<span class="go">                        &quot;body&quot;</span>
<span class="go">                    ]</span>
<span class="go">                }</span>
<span class="go">            },</span>
<span class="go">            {</span>
<span class="go">                &quot;description&quot;: &quot;Parsed field&quot;,</span>
<span class="go">                &quot;inputs&quot;: {</span>
<span class="go">                    &quot;fields&quot;: [</span>
<span class="go">                        {</span>
<span class="go">                            &quot;name&quot;: &quot;body&quot;,</span>
<span class="go">                            &quot;origin&quot;: &quot;File.Read&quot;</span>
<span class="go">                        }</span>
<span class="go">                    ]</span>
<span class="go">                },</span>
<span class="go">                &quot;name&quot;: &quot;CSVParser.CSV Parse&quot;,</span>
<span class="go">                &quot;outputs&quot;: {</span>
<span class="go">                    &quot;fields&quot;: [</span>
<span class="go">                        &quot;id&quot;,</span>
<span class="go">                        &quot;first_name&quot;,</span>
<span class="go">                        &quot;last_name&quot;</span>
<span class="go">                    ]</span>
<span class="go">                }</span>
<span class="go">            },</span>
<span class="go">            {</span>
<span class="go">                &quot;description&quot;: &quot;Used as a key in a join&quot;,</span>
<span class="go">                &quot;inputs&quot;: {</span>
<span class="go">                    &quot;fields&quot;: [</span>
<span class="go">                        {</span>
<span class="go">                            &quot;name&quot;: &quot;id&quot;,</span>
<span class="go">                            &quot;origin&quot;: &quot;CSVParser.CSV Parse&quot;</span>
<span class="go">                        },</span>
<span class="go">                        {</span>
<span class="go">                            &quot;name&quot;: &quot;customer_id&quot;,</span>
<span class="go">                            &quot;origin&quot;: &quot;CSVParser2.CSV Parse&quot;</span>
<span class="go">                        }</span>
<span class="go">                    ]</span>
<span class="go">                },</span>
<span class="go">                &quot;name&quot;: &quot;Joiner.Join&quot;,</span>
<span class="go">                &quot;outputs&quot;: {</span>
<span class="go">                    &quot;fields&quot;: [</span>
<span class="go">                        &quot;id&quot;,</span>
<span class="go">                        &quot;customer_id&quot;</span>
<span class="go">                    ]</span>
<span class="go">                }</span>
<span class="go">            },</span>
<span class="go">            {</span>
<span class="go">                &quot;description&quot;: &quot;Wrote to TPFS dataset&quot;,</span>
<span class="go">                &quot;inputs&quot;: {</span>
<span class="go">                    &quot;fields&quot;: [</span>
<span class="go">                        {</span>
<span class="go">                            &quot;name&quot;: &quot;id_from_customer&quot;,</span>
<span class="go">                            &quot;origin&quot;: &quot;Joiner.Rename id&quot;</span>
<span class="go">                        },</span>
<span class="go">                        {</span>
<span class="go">                            &quot;name&quot;: &quot;fname&quot;,</span>
<span class="go">                            &quot;origin&quot;: &quot;Joiner.Rename CSVParser.first_name&quot;</span>
<span class="go">                        },</span>
<span class="go">                        {</span>
<span class="go">                            &quot;name&quot;: &quot;lname&quot;,</span>
<span class="go">                            &quot;origin&quot;: &quot;Joiner.Rename CSVParser.last_name&quot;</span>
<span class="go">                        },</span>
<span class="go">                        {</span>
<span class="go">                            &quot;name&quot;: &quot;customer_id&quot;,</span>
<span class="go">                            &quot;origin&quot;: &quot;Joiner.Join&quot;</span>
<span class="go">                        },</span>
<span class="go">                        {</span>
<span class="go">                            &quot;name&quot;: &quot;item&quot;,</span>
<span class="go">                            &quot;origin&quot;: &quot;Joiner.Identity CSVParser2.item&quot;</span>
<span class="go">                        }</span>
<span class="go">                    ]</span>
<span class="go">                },</span>
<span class="go">                &quot;name&quot;: &quot;Parquet Time Partitioned Dataset.Write&quot;,</span>
<span class="go">                &quot;outputs&quot;: {</span>
<span class="go">                    &quot;endPoint&quot;: {</span>
<span class="go">                        &quot;name&quot;: &quot;parquet_data&quot;,</span>
<span class="go">                        &quot;namespace&quot;: &quot;default&quot;</span>
<span class="go">                    }</span>
<span class="go">                }</span>
<span class="go">            }</span>
<span class="go">        ],</span>
<span class="go">        &quot;programs&quot;: [</span>
<span class="go">            {</span>
<span class="go">                &quot;lastExecutedTimeInSeconds&quot;: 1532468358,</span>
<span class="go">                &quot;program&quot;: {</span>
<span class="go">                    &quot;application&quot;: &quot;customer_pipeline_spark&quot;,</span>
<span class="go">                    &quot;entity&quot;: &quot;PROGRAM&quot;,</span>
<span class="go">                    &quot;namespace&quot;: &quot;default&quot;,</span>
<span class="go">                    &quot;program&quot;: &quot;DataPipelineWorkflow&quot;,</span>
<span class="go">                    &quot;type&quot;: &quot;Workflow&quot;,</span>
<span class="go">                    &quot;version&quot;: &quot;-SNAPSHOT&quot;</span>
<span class="go">                }</span>
<span class="go">            }</span>
<span class="go">        ]</span>
<span class="go">    }</span>
<span class="go">  ]</span>
<span class="go">}</span>
</pre></div>
</div>
<p class="rubric">HTTP Responses</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Status Codes</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">200</span> <span class="pre">OK</span></code></td>
<td>Fields of dataset are returned as a list of strings in the body of the response</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">400</span> <span class="pre">BAD</span> <span class="pre">REQUEST</span></code></td>
<td>Failure to parse the time range provided</td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="section" id="retrieving-metadata-for-a-program-run-deprecated">
<span id="http-restful-api-metadata-run"></span><h2>Retrieving Metadata for a Program Run (Deprecated)<a class="headerlink" href="#retrieving-metadata-for-a-program-run-deprecated" title="Permalink to this headline">🔗</a></h2>
<p>At every run of a program, the metadata associated with the program, the application it is part of, and any datasets
and streams used by the program run are recorded. To retrieve the metadata for a program run, submit an HTTP GET request:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">GET /v3/namespaces/&lt;namespace-id&gt;/apps/&lt;app-id&gt;/&lt;program-type&gt;/&lt;program-id&gt;/runs/&lt;run-id&gt;/metadata</span>
</pre></div>
</div>
<p>with the metadata returned as a JSON string in the return body:</p>
<div class="highlight-json-ellipsis notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
    <span class="p">{</span>
        <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
            <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;STREAM&quot;</span><span class="p">,</span>
            <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span><span class="p">,</span>
            <span class="nt">&quot;stream&quot;</span><span class="p">:</span> <span class="s2">&quot;purchaseStream&quot;</span>
        <span class="p">},</span>
        <span class="nt">&quot;properties&quot;</span><span class="p">:</span> <span class="p">{},</span>
        <span class="nt">&quot;scope&quot;</span><span class="p">:</span> <span class="s2">&quot;USER&quot;</span><span class="p">,</span>
        <span class="nt">&quot;tags&quot;</span><span class="p">:</span> <span class="p">[]</span>
    <span class="p">},</span>
    <span class="p">{</span>
        <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
            <span class="nt">&quot;application&quot;</span><span class="p">:</span> <span class="s2">&quot;PurchaseHistory&quot;</span><span class="p">,</span>
            <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;PROGRAM&quot;</span><span class="p">,</span>
            <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span><span class="p">,</span>
            <span class="nt">&quot;program&quot;</span><span class="p">:</span> <span class="s2">&quot;PurchaseFlow&quot;</span><span class="p">,</span>
            <span class="nt">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;Flow&quot;</span><span class="p">,</span>
            <span class="nt">&quot;version&quot;</span><span class="p">:</span> <span class="s2">&quot;-SNAPSHOT&quot;</span>
        <span class="p">},</span>
        <span class="nt">&quot;properties&quot;</span><span class="p">:</span> <span class="p">{},</span>
        <span class="nt">&quot;scope&quot;</span><span class="p">:</span> <span class="s2">&quot;USER&quot;</span><span class="p">,</span>
        <span class="nt">&quot;tags&quot;</span><span class="p">:</span> <span class="p">[</span>
            <span class="s2">&quot;flow-tag1&quot;</span>
        <span class="p">]</span>
    <span class="p">},</span>
    <span class="p">{</span>
        <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
            <span class="nt">&quot;dataset&quot;</span><span class="p">:</span> <span class="s2">&quot;purchases&quot;</span><span class="p">,</span>
            <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;DATASET&quot;</span><span class="p">,</span>
            <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span>
        <span class="p">},</span>
        <span class="nt">&quot;properties&quot;</span><span class="p">:</span> <span class="p">{},</span>
        <span class="nt">&quot;scope&quot;</span><span class="p">:</span> <span class="s2">&quot;USER&quot;</span><span class="p">,</span>
        <span class="nt">&quot;tags&quot;</span><span class="p">:</span> <span class="p">[]</span>
    <span class="p">},</span>
    <span class="p">{</span>
        <span class="nt">&quot;entityId&quot;</span><span class="p">:</span> <span class="p">{</span>
            <span class="nt">&quot;application&quot;</span><span class="p">:</span> <span class="s2">&quot;PurchaseHistory&quot;</span><span class="p">,</span>
            <span class="nt">&quot;entity&quot;</span><span class="p">:</span> <span class="s2">&quot;APPLICATION&quot;</span><span class="p">,</span>
            <span class="nt">&quot;namespace&quot;</span><span class="p">:</span> <span class="s2">&quot;default&quot;</span><span class="p">,</span>
            <span class="nt">&quot;version&quot;</span><span class="p">:</span> <span class="s2">&quot;-SNAPSHOT&quot;</span>
        <span class="p">},</span>
        <span class="nt">&quot;properties&quot;</span><span class="p">:</span> <span class="p">{},</span>
        <span class="nt">&quot;scope&quot;</span><span class="p">:</span> <span class="s2">&quot;USER&quot;</span><span class="p">,</span>
        <span class="nt">&quot;tags&quot;</span><span class="p">:</span> <span class="p">[</span>
            <span class="s2">&quot;app-tag1&quot;</span>
        <span class="p">]</span>
    <span class="p">}</span>
<span class="p">]</span>
</pre></div>
</div>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">namespace-id</span></code></td>
<td>Namespace ID</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">app-id</span></code></td>
<td>Name of the application</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">program-type</span></code></td>
<td>One of <code class="docutils literal notranslate"><span class="pre">flows</span></code>, <code class="docutils literal notranslate"><span class="pre">mapreduce</span></code>, <code class="docutils literal notranslate"><span class="pre">spark</span></code>, <code class="docutils literal notranslate"><span class="pre">workflows</span></code>, <code class="docutils literal notranslate"><span class="pre">services</span></code>, or <code class="docutils literal notranslate"><span class="pre">workers</span></code></td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">program-id</span></code></td>
<td>Name of the program</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">run-id</span></code></td>
<td>Program run id</td>
</tr>
</tbody>
</table>
<p class="rubric">HTTP Responses</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Status Codes</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">200</span> <span class="pre">OK</span></code></td>
<td>The properties requested were returned as a JSON string in the body of the response</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">404</span> <span class="pre">NOT</span> <span class="pre">FOUND</span></code></td>
<td>The entity, program, or run for which properties are being requested was not found</td>
</tr>
</tbody>
</table>
<p><strong>Note</strong>: The Metadata aggregation of program, application, stream and dataset associated with a program is
deprecated. Program runs are treated like any other entity to which metadata can be associated directly.
To retrieve the direct metadata of program run please refer to
<a class="reference internal" href="#http-restful-api-metadata-properties"><span class="std std-ref">Metadata Properties</span></a> and
<a class="reference internal" href="#http-restful-api-metadata-tags"><span class="std std-ref">Metadata Tags</span></a>. An additional query parameter <code class="docutils literal notranslate"><span class="pre">aggregateRun</span></code> must be set to
<code class="docutils literal notranslate"><span class="pre">false</span></code> while operating with metadata directly associated with program runs.</p>
</div>
<div class="section" id="metadata-for-custom-entities">
<span id="metadata-custom-entities"></span><h2>Metadata for Custom Entities<a class="headerlink" href="#metadata-for-custom-entities" title="Permalink to this headline">🔗</a></h2>
<p>Metadata can also be associated with custom entities. In CDAP Entities are separated into two kinds:</p>
<ul class="simple">
<li>CDAP Entities: These are system defined entities that have special meaning in CDAP. <code class="docutils literal notranslate"><span class="pre">Namespace</span></code>, <code class="docutils literal notranslate"><span class="pre">Stream</span></code>, <code class="docutils literal notranslate"><span class="pre">Application</span></code>, <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> etc. are example of CDAP Entities.</li>
<li>Custom Entities: These are user defined entities that represent a resource that exists in CDAP.</li>
</ul>
<p>Custom Entities are represented as a hierarchical key-value pair and can optionally have a explicitly defined type.</p>
<p>If a type is not specified then the last key in the hierarchy is considered as the type.</p>
<p>In REST APIs, custom entities are represented as hierarchical key-value pairs. If the last key in the hierarchy is
not a type, the type is specified as a query parameter.</p>
<p>For example, to add tags to a custom file entity in a dataset:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">POST /v3/namespaces/&lt;namespace-id&gt;/datasets/&lt;dataset-id&gt;/file/&lt;file-name&gt;/metadata/tags</span>
</pre></div>
</div>
<p>In the example above, the custom entity is a single key-value pair where <code class="docutils literal notranslate"><span class="pre">file</span></code> is the key and <code class="docutils literal notranslate"><span class="pre">&lt;file-name&gt;</span></code>
is the value.</p>
<p>To add tags to a custom jar entity in a namespace:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">POST /v3/namespaces/&lt;namespace-id&gt;/jar/&lt;jar-id&gt;/versions/&lt;jar-version&gt;/metadata/tags[?type=jar]</span>
</pre></div>
</div>
<p>In the example above, the custom entity consists of two key-value pairs. The first has key <code class="docutils literal notranslate"><span class="pre">jar</span></code> and value
<code class="docutils literal notranslate"><span class="pre">&lt;jar-id&gt;</span></code>. The second has key <code class="docutils literal notranslate"><span class="pre">versions</span></code> and value <code class="docutils literal notranslate"><span class="pre">&lt;jar-version&gt;</span></code>. We pass the jar as the type to specify
the type of the entity since the last key in the hierarchy is not the type in this case.</p>
<p>In both examples, the metadata tags are passed as a JSON list of strings in the request body:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">[&quot;tag1&quot;, &quot;tag2&quot;]</span>
</pre></div>
</div>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Logging HTTP RESTful API" href="logging.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="Metrics HTTP RESTful API" href="metrics.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../_static/js/js.cookie.js"></script><script type="text/javascript" src="../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>